import Queue
import threading
import urllib2
import time
from BeautifulSoup import BeautifulSoup

hosts = ["http://outlook.com","http://sina.com.cn", "http://zhaopin.com","http://51job.com","http://91cha.com","http://qycn.com","http://aizhan.com","http://7c.com", "http://taobao.com", "http://360buy.com",
         "http://hao123.com", "http://265.com", "http://zhihu.com"]

queue = Queue.Queue()
out_queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    def __init__(self, queue, out_queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.out_queue = out_queue

    
    def run(self):
        while True:
            host = self.queue.get()

            url = urllib2.urlopen(host)
            chunk = url.read()

            #place chunk into out queue
            self.out_queue.put(chunk)

            self.queue.task_done()

class DatamineThread(threading.Thread):
    def __init__(self, out_queue):
        threading.Thread.__init__(self)
        self.out_queue = out_queue

    def run(self):
        chunk = self.out_queue.get()

        soup = BeautifulSoup(chunk)
        print soup.findAll(['title'])

        self.out_queue.task_done()
        
start = time.time()
totals = len(hosts)
print totals
def main():
    for i in range(totals):
        t = ThreadUrl(queue, out_queue)
        t.setDaemon(True)
        t.start()

    for host in hosts:
        queue.put(host)

    for i in range(totals):
        dt = DatamineThread(out_queue)
        dt.setDaemon(True)
        dt.start()

    queue.join()
    out_queue.join()

main()

print "Elapsed Time: %s " % (time.time() - start)
